#!/usr/bin/ruby
$: << File.join(File.expand_path(File.dirname(__FILE__)), "..")
require 'rubygems'
require 'optparse'
require 'EasyHadoop'

# Normalizer
module Stemmable

  STEP_2_LIST = {
    'ational'=>'ate', 'tional'=>'tion', 'enci'=>'ence', 'anci'=>'ance',
    'izer'=>'ize', 'bli'=>'ble',
    'alli'=>'al', 'entli'=>'ent', 'eli'=>'e', 'ousli'=>'ous',
    'ization'=>'ize', 'ation'=>'ate',
    'ator'=>'ate', 'alism'=>'al', 'iveness'=>'ive', 'fulness'=>'ful',
    'ousness'=>'ous', 'aliti'=>'al',
    'iviti'=>'ive', 'biliti'=>'ble', 'logi'=>'log'
  }
  
  STEP_3_LIST = {
    'icate'=>'ic', 'ative'=>'', 'alize'=>'al', 'iciti'=>'ic',
    'ical'=>'ic', 'ful'=>'', 'ness'=>''
  }


  SUFFIX_1_REGEXP = /(
                    ational  |
                    tional   |
                    enci     |
                    anci     |
                    izer     |
                    bli      |
                    alli     |
                    entli    |
                    eli      |
                    ousli    |
                    ization  |
                    ation    |
                    ator     |
                    alism    |
                    iveness  |
                    fulness  |
                    ousness  |
                    aliti    |
                    iviti    |
                    biliti   |
                    logi)$/x


  SUFFIX_2_REGEXP = /(
                      al       |
                      ance     |
                      ence     |
                      er       |
                      ic       | 
                      able     |
                      ible     |
                      ant      |
                      ement    |
                      ment     |
                      ent      |
                      ou       |
                      ism      |
                      ate      |
                      iti      |
                      ous      |
                      ive      |
                      ize)$/x


  C = "[^aeiou]"         # consonant
  V = "[aeiouy]"         # vowel
  CC = "#{C}(?>[^aeiouy]*)"  # consonant sequence
  VV = "#{V}(?>[aeiou]*)"    # vowel sequence

  MGR0 = /^(#{CC})?#{VV}#{CC}/o                # [cc]vvcc... is m>0
  MEQ1 = /^(#{CC})?#{VV}#{CC}(#{VV})?$/o       # [cc]vvcc[vv] is m=1
  MGR1 = /^(#{CC})?#{VV}#{CC}#{VV}#{CC}/o      # [cc]vvccvvcc... is m>1
  VOWEL_IN_STEM   = /^(#{CC})?#{V}/o                      # vowel in stem

  # PORTER STEMMER - ADOPTED FROM http://tartarus.org/~martin/PorterStemmer/ruby.txt
  
  def stem_porter

    # make a copy of the given object and convert it to a string.
    w = self.dup.to_str
    
    return w if w.length < 3
    
    # now map initial y to Y so that the patterns never treat it as vowel
    w[0] = 'Y' if w[0] == ?y
    
    # Step 1a
    if w =~ /(ss|i)es$/
      w = $` + $1
    elsif w =~ /([^s])s$/ 
      w = $` + $1
    end

    # Step 1b
    if w =~ /eed$/
      w.chop! if $` =~ MGR0 
    elsif w =~ /(ed|ing)$/
      stem = $`
      if stem =~ VOWEL_IN_STEM 
        w = stem
	case w
        when /(at|bl|iz)$/             then w << "e"
        when /([^aeiouylsz])\1$/       then w.chop!
        when /^#{CC}#{V}[^aeiouwxy]$/o then w << "e"
        end
      end
    end

    if w =~ /y$/ 
      stem = $`
      w = stem + "i" if stem =~ VOWEL_IN_STEM 
    end

    # Step 2
    if w =~ SUFFIX_1_REGEXP
      stem = $`
      suffix = $1
      # print "stem= " + stem + "\n" + "suffix=" + suffix + "\n"
      if stem =~ MGR0
        w = stem + STEP_2_LIST[suffix]
      end
    end

    # Step 3
    if w =~ /(icate|ative|alize|iciti|ical|ful|ness)$/
      stem = $`
      suffix = $1
      if stem =~ MGR0
        w = stem + STEP_3_LIST[suffix]
      end
    end

    # Step 4
    if w =~ SUFFIX_2_REGEXP
      stem = $`
      if stem =~ MGR1
        w = stem
      end
    elsif w =~ /(s|t)(ion)$/
      stem = $` + $1
      if stem =~ MGR1
        w = stem
      end
    end

    #  Step 5
    if w =~ /e$/ 
      stem = $`
      if (stem =~ MGR1) ||
          (stem =~ MEQ1 && stem !~ /^#{CC}#{V}[^aeiouwxy]$/o)
        w = stem
      end
    end

    if w =~ /ll$/ && w =~ MGR1
      w.chop!
    end

    # and turn initial Y back to y
    w[0] = 'y' if w[0] == ?Y

    w
  end

  #Porter 
  def porter
	input = self.dup.to_str
	output = input.to_s.strip.split(/\s/)
	tokens = []
	output.each do |token|
		tokens << token.stem_porter unless token.nil? or token.to_s.strip.length==0
	end
	return tokens.join(" ")

  end

  # Alpha Numeric 
  def alpha_numeric
	input = self.dup.to_str
        output = input.to_s.strip.gsub(/[-_]/,' ').split(/\s/)
        tokens = []
        output.each do |token|
                token = token.to_s.gsub(/[^[:alnum:]]/,'')
                next if token.strip.length == 0
                tokens << token
        end
        output = tokens.join(" ")
        return(output)	
  end
  
 
end

#Include Stemmable Module
class String
	include Stemmable 
end

# Define Class
class Normalizer < EasyHadoop
	
	@@NORMALIZER_OPERATIONS = ["porter", "downcase", "alphanum", "nodup", "sort"]
	attr_reader :field_keys

	#constructor
	def initialize(arguments)
		super(arguments)
		@field_keys = @options[:fields].keys
	end
  
 	def set_mandatory
		super
		@mandatory << :fields
	end
	
	#handle command line arguments
	def set_commands
		super
		@optparse.banner = "Usage: ruby #{__FILE__} --field COL:NORMALIZER --field col:NORMALIZER"

		@optparse.on('-f','--field COL:FUNC_LIST', 
				"Specify column index number on which to apply the normalization functions. Currently available normalization functions are #{@@NORMALIZER_OPERATIONS.join(', ')}") do |value|
				@options[:fields] = {} if @options[:fields].nil?
				col,normalizers = value.split(':')
				begin
					col = Integer(col)
				rescue
					raise OptionParser::OptionParser::ParseError, "Unable to cast #{col} to Integer"
				end
				@options[:fields][col.to_i] = []
				@options[:fields][col] = OptionParserUtil.handle_operations(
						normalizers, @@NORMALIZER_OPERATIONS)
		end
		
		@options[:retain] = false
		@optparse.on('-r','--retain', 'Retain original values'){@options[:retain] = true}
		
	end

  	def execute(line)
  		
  		return nil if line.nil? or line.chomp.empty?
  		
		tokens = line.to_s.chomp.split(@options[:input_delimiter])
		token_length = tokens.length
		@field_keys.each{|f|
			value = tokens[f]
			
			#sanity check
			if value.nil? or value.length == 0
				tokens << ""
				next
			end 
			
			normalizers = @options[:fields][f]
			normalizers.each{|n|
		  		value = value.downcase if (n=="downcase")
		  		value = value.alpha_numeric if(n=="alphanum")
		  		value = value.porter if(n=="porter")
		  		value = value.split(/\s{1,}/).uniq.join(" ") if(n=="nodup")
		  		value = value.split(/\s{1,}/).sort.join(" ") if(n=="sort")
			}
			tokens << value
		}
		
		return tokens.join(@options[:output_delimiter]) if @options[:retain]
		
		# if retain is false
		@field_keys.each_index{|index|
			tokens[@field_keys[index]] = tokens[token_length + index ]
		}
		return tokens[0..token_length-1].join(@options[:output_delimiter])
		
		   
  	end
end

# Scripting Option
if __FILE__ == $0
    cls = Normalizer.new(ARGV)
    STDIN.each_line{|line|
	  value = cls.execute(line)
      puts value unless value.nil?
    }
end



